﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using rtw;
namespace stats
{
   
    class Program
    {
        public static void log(string mes)
        {
            Console.WriteLine(mes);
        }
        /*
        class daystat       
        {
            public Dictionary<int, hourstat> hours = new Dictionary<int, hourstat>();
        }
        class hourstat
        {
           public  int totaltwits;
        }

        class monthstat
        {
            public Dictionary<int, daystat> days = new Dictionary<int, daystat>();
        }

        class yearstat2
        {
            public Dictionary<int, monthstat> months = new Dictionary<int, monthstat>();
        }
        */
        static void logstat(string name,   int all,   List<entry>  part, int totalHours)
        {
            int cc=part.ToList().Count ;
            double spampercent = ((double)cc/ all) * 100f;
            double twitsPerday = ((double)cc / (double)totalHours)*24;
            log(name + ": " + string.Format("{0:00}%", spampercent) + "   " + cc + " twits " + string.Format("{0:00}", twitsPerday)+ " per day");
        }
        static void Main(string[] args)
        {
            log("Load...");

            string fetcheddata = System.IO.Path.Combine("../", "Data/1scaner");
            var entries = loader.all(fetcheddata);
            log("Remove duplicates...");
            var hashes = new Dictionary<string,entry>();
            foreach (var oe in entries)
            {
                if(!hashes.ContainsKey(oe.title))
                    hashes.Add(oe.title, oe);
            }
            var cleared = new List<entry>();
            foreach (var kvp in hashes)
            {
                cleared.Add(kvp.Value);
            }
            /*
            
            foreach (var oe in entries)
            {
                bool ex = cleared.Exists((r) => r.title == oe.title);
                if (!ex)
                    cleared.Add(oe);
            }*/
            log("  removed duplicates: " + (entries.Count - cleared.Count));

            entries = cleared;
            rtw.entrywriter.set_data_folder("2filter");

           DateTime start = DateTime.Parse(entries[0].published);
           DateTime end = DateTime.Parse(entries[entries.Count-1].published);
           int totalhoursInSTats = (int)((end - start).TotalHours);

           
            int TOTAL=entries.Count;
            log("Loaded: " + string.Format("{0:00}", totalhoursInSTats) + " hours " + TOTAL+ " twits");
            /*
           var byauthors = from e in entries
                              group e by e.author into byauthor
                          select byauthor;

           var spammers = from s in byauthors
                              where s.Count()>10
                          select s;

            //dump spammers
           List<entry> spamtwits = new List<entry>();


          

           List<string> spammersNames = new List<string>();
           var sl=spammers.ToList();
           sl.ForEach((i) => spammersNames.Add(i.Key));


           var spam = from e in entries
                         where (spammersNames.Contains(e.author))
                         select e;

           logstat("SPAM", TOTAL, spam.ToList(), totalhoursInSTats);
           entrywriter.write_entries("spam",true,spam.ToList());
           

           entries=removeIntersection(entries, spam.ToList());
            */
           //какой процент оказался спамом
 
           



            List<string> whiteList = new List<string>()
            {
                "car","leg","brok","help","pray","dead","inj",
                "feel","gma","son","child","father","dude","mother","mom",
                "horrible","hurt", "god","?","fuck","i ",
                "mouth","break","crie","bone",
                "tell","lost","arm","just",
                "worri"
            };

            List<string> blackList = new List<string>()
            {
                "delet","news","erased","read","report","accident:", "mva","(minor)"
            };

            List<string> blackList2 = new List<string>()
            {
                "awyer","ttorney","ttorney","travolta","i-","news","nsurance",
                "major accident","traffic update"
            };
            List<string> authors_black_list = new List<string>()
          {
              "news","traffic","road","local","local","updates"
          };



            foreach (var e in entries)
            {
                foreach (var w in whiteList)
                    if (e.title_lowered.IndexOf(w) > 0)
                        e.rank++;

                foreach (var b in blackList)
                   if (e.title_lowered.IndexOf(b) > 0)
                        e.rank--;

                foreach (var b in blackList2)
                    if (e.title_lowered.IndexOf(b) > 0)
                        e.rank-=5;

                foreach (var b in authors_black_list)
                    if (e.author.IndexOf(b) > 0)
                        e.rank -= 5;
            }

    

            for(int score=-1000;score<1000;score++)
            {
            var best = from e in entries
                       where e.rank>=score && e.rank<(score+1)
                       select e;
                     

                     if (best.ToList().Count > 0)
                     {
                         string folder="3best";
                         if(score<0)
                             folder="2filter";
                         save_scored(best.ToList(), score, TOTAL, totalhoursInSTats, folder);
                         entries = removeIntersection(entries, best.ToList());
                     }
            }

            logstat("other", TOTAL, entries, totalhoursInSTats);
            entrywriter.write_entries("other", true, entries);

        }
        static void save_scored(List<entry> best, int score, int TOTAL, int totalhoursInSTats, string folder)
        {
            logstat("best" + score, TOTAL, best.ToList(), totalhoursInSTats);

            string bestfn = "best" + score;
            bestfn = System.IO.Path.Combine("../Data/" + folder, bestfn);

            

            entrywriter.write_entries_onefile(bestfn, true, best.ToList());

        }
        static List<entry> removeIntersection(List<entry> e1, List<entry> e2)
        {
            List<entry> res=e1;
            e2.ForEach((e) => e1.Remove(e));
            return res;
        }
     /*  static hourstat gethour( yearstat ys, int month, int day, int hour)
        {
            
            monthstat m;
            if (!ys.months.TryGetValue(month, out m))
            {
                m = new monthstat();
                ys.months.Add(month,m);
            }
            daystat d;
            if (!m.days.TryGetValue(day, out d))
            {
                d= new daystat();
                m.days.Add(day, d);
            }
            hourstat res;
            if (!d.hours.TryGetValue(hour, out res))
            {
                res = new hourstat();
                d.hours.Add(hour, res);
            }

            return res;
        }
   */ }
}
